"""
This code is based on the tutorial from deeplearning.org


References:

    - textbooks: "Pattern Recognition and Machine Learning" -
                 Christopher M. Bishop, section 5

"""
__docformat__ = 'restructedtext en'


import cPickle
import gzip
import os
import sys
import time

import numpy

import theano
import theano.tensor as T
from theano import sparse as S
import nnutils
from weight_init_strategy import WeightInit


class HiddenLayer(object):
    def __init__(self, numpy_rng, theano_rng, input,
                 n_in, n_out, W=None, b=None,
                 activation=T.tanh, L1rec=None, L2rec=None, 
                 dropout_rate=None, 
                 init_strategy=WeightInit):
        """
        Typical hidden layer of a MLP: units are fully-connected and have
        sigmoidal activation function. Weight matrix W is of shape (n_in,n_out)
        and the bias vector b is of shape (n_out,).

        NOTE : The nonlinearity used here is tanh

        Hidden unit activation is given by: tanh(dot(input,W) + b)

        :type rng: numpy.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type input: theano.tensor.dmatrix
        :param input: a symbolic tensor of shape (n_examples, n_in)

        :type n_in: int
        :param n_in: dimensionality of input

        :type n_out: int
        :param n_out: number of hidden units

        :type activation: theano.Op or function
        :param activation: Non linearity to be applied in the hidden
                           layer
        """
        self.input = input
        self.dropout_rate = dropout_rate
        # `W` is initialized with `W_values` which is uniformely sampled
        # from sqrt(-6./(n_in+n_hidden)) and sqrt(6./(n_in+n_hidden))
        # for tanh activation function
        # the output of uniform if converted using asarray to dtype
        # theano.config.floatX so that the code is runable on GPU
        # Note : optimal initialization of weights is dependent on the
        #        activation function used (among other things).
        #        For example, results presented in [Xavier10] suggest that you
        #        should use 4 times larger initial weights for sigmoid
        #        compared to tanh
        #        We have no info for other function, so we use the same as
        #        tanh.
        if W is None:
            W_values = init_strategy.init()
            if activation == theano.tensor.tanh:
                W_values *= 0.25
            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            if activation == theano.tensor.tanh:
                b_values = numpy.ones((n_out,), dtype=theano.config.floatX)*0.5
            else:
                b_values = numpy.zeros((n_out,), dtype=theano.config.floatX)
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b
        
        if isinstance(input.type, S.SparseType):
            lin_output = S.dot(input, self.W) + self.b
        else:
            if dropout_rate == None:
                lin_output = T.dot(input, self.W) + self.b
                lin_output_dropout = lin_output
            else:
                h_dropout = T.dot(input \
                            * T.cast(
                                     theano_rng.binomial(size=input.shape, 
                                                         n=1, p=1-dropout_rate
                                        ), 
                                     dtype=theano.config.floatX
                                     ), \
                            self.W)
                
                lin_output_dropout = h_dropout + self.b
                lin_output = T.dot(input, self.W*(1-dropout_rate)) + self.b
        
        self.output = (lin_output if activation is None
                       else activation(lin_output))
        self.output_dropout = (lin_output_dropout if activation is None
                       else activation(lin_output_dropout))
        # parameters of the model
        self.params = [self.W, self.b]
        
        if L1rec == None:
            self.L1rec = abs(self.W).sum()
        else:
            self.L1rec = L1rec + abs(self.W).sum()
            
        
        if L2rec == None:
            self.L2rec = (self.W **2).sum()
        else:
            self.L2rec = L2rec + (self.W **2).sum()
